class RecruitSpider(scrapy.Spider):
    name = "recruit"
    allowed_domains = ["www.techlabplt.com"]
    start_urls = ["http://www.techlabplt.com:8082/searchNew?orderBy=releaseDate&keyword=JAVA",
                  "http://www.techlabplt.com:8082/searchNew?orderBy=releaseDate&keyword=PHP",
                  "http://www.techlabplt.com:8082/searchNew?orderBy=releaseDate&keyword=大数据"]
 
    # 重写start_requests，前置请求，用于访问登录界面，存储cookie
    def start_requests(self):
        login_url = "http://www.techlabplt.com:8082/user/login"
        yield scrapy.Request(url=login_url)
 
    # 登录成功后，发起对数据的请求
    def parse(self, response):
        # 循环遍历三个URL
        for url in self.start_urls:
            # 取50页数据
            for num in range(1, 51):
                time.sleep(2)
                print(url + "--------------------" + str(num))
                page_url = url + "&pageNum=" + str(num)
                print("发起请求")
                yield scrapy.Request(url=page_url, callback=self.data_analysis, dont_filter=True)
 
    # 解析页面数据
    def data_analysis(self, response):
        itm = RecruitscrapyItem()
        # 返回的数据转化为json格式,并获取data信息
        data = response.json()['data']
        # print("json数据------------------：" + str(data))
 
        for i in range(len(data)):
            print("data:----" + str(data[i]['title']))
            # 将各个字段提取出来，存入itm
            itm['recruit_name'] = data[i]['title']
            itm['salary_down'] = data[i]['salaryDown']
            itm['salary_up'] = data[i]['salaryUp']
            itm['work_city'] = data[i]['workCity']
            itm['quantity'] = data[i]['quantity']
            itm['release_time'] = data[i]['releaseDate']
            itm['company'] = data[i]['companyName']
            # gbk编码问题，作手动转义
            itm['requirement'] = data[i]['requirement'].replace('\xa0', "\n")
            # 运行过程中，发现有些职位下没有skill，所以加判断是否存在此key
            if 'skill' in data[i]:
                itm['skill'] = data[i]['skill']
            # 返回数据
            yield itm
